import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;

import org.apache.log4j.Logger;

import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.tdb.TDBFactory;


public class TDBloader {
	
	public static void main (String[] args) throws IOException {
		File file = new File("..\\..\\n-triples\\location.nt");
		BufferedReader br = new BufferedReader(new FileReader(file));
		String ntriple;
		Logger logger = Logger.getLogger(TDBloader.class);
		
		Dataset ds = TDBFactory.createDataset("..\\..\\n-triples\\merged");
		Model model = ds.getDefaultModel();
		Model tmpModel = ModelFactory.createDefaultModel();
		
		int i = 1;
		int total = 0;
		
		while((ntriple = br.readLine()) != null) {
			//convert string to input stream for jena to parse
			byte[] bytes;
			//try to convert to utf-8
			try {
				bytes = ntriple.getBytes("UTF-8");
			} catch (UnsupportedEncodingException e) {
			//if conversion fails, replace every non utf-8 char with a questionmark
				ntriple.replaceAll("[^(\\x20-\\x7F)]", "?");
				bytes = ntriple.getBytes();
			}
			InputStream is = new ByteArrayInputStream(bytes);
			try {
				tmpModel.read(is, null, "N-TRIPLE");
			}
			catch (Exception e) {
//				e.printStackTrace();
				String output = "Error: " + ntriple + "(line: " + (total + i) + ")";
				System.out.println(output);
				logger.error(output);
			}
			
			if(i >= 50000) {
				model.add(tmpModel);
				tmpModel = ModelFactory.createDefaultModel();
				total += i;
				i = 0;
				System.out.println(total);
			}
			i++;
		}
	}
}
